{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from numpy import ndarray\n",
    "\n",
    "from typing import List"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def assert_same_shape(array: ndarray,\n",
    "                      array_grad: ndarray):\n",
    "    assert array.shape == array_grad.shape, \\\n",
    "        '''\n",
    "        Two ndarrays should have the same shape;\n",
    "        instead, first ndarray's shape is {0}\n",
    "        and second ndarray's shape is {1}.\n",
    "        '''.format(tuple(array_grad.shape), tuple(array.shape))\n",
    "    return None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `Operation` and `ParamOperation`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Operation(object):\n",
    "    '''\n",
    "    Base class for an \"operation\" in a neural network.\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "    def forward(self, input_: ndarray):\n",
    "        '''\n",
    "        Stores input in the self._input instance variable\n",
    "        Calls the self._output() function.\n",
    "        '''\n",
    "        self.input_ = input_\n",
    "\n",
    "        self.output = self._output()\n",
    "\n",
    "        return self.output\n",
    "\n",
    "\n",
    "    def backward(self, output_grad: ndarray) -> ndarray:\n",
    "        '''\n",
    "        Calls the self._input_grad() function.\n",
    "        Checks that the appropriate shapes match.\n",
    "        '''\n",
    "        assert_same_shape(self.output, output_grad)\n",
    "\n",
    "        self.input_grad = self._input_grad(output_grad)\n",
    "\n",
    "        assert_same_shape(self.input_, self.input_grad)\n",
    "        return self.input_grad\n",
    "\n",
    "\n",
    "    def _output(self) -> ndarray:\n",
    "        '''\n",
    "        The _output method must be defined for each Operation\n",
    "        '''\n",
    "        raise NotImplementedError()\n",
    "\n",
    "\n",
    "    def _input_grad(self, output_grad: ndarray) -> ndarray:\n",
    "        '''\n",
    "        The _input_grad method must be defined for each Operation\n",
    "        '''\n",
    "        raise NotImplementedError()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ParamOperation(Operation):\n",
    "    '''\n",
    "    An Operation with parameters.\n",
    "    '''\n",
    "\n",
    "    def __init__(self, param: ndarray) -> ndarray:\n",
    "        '''\n",
    "        The ParamOperation method\n",
    "        '''\n",
    "        super().__init__()\n",
    "        self.param = param\n",
    "\n",
    "    def backward(self, output_grad: ndarray) -> ndarray:\n",
    "        '''\n",
    "        Calls self._input_grad and self._param_grad.\n",
    "        Checks appropriate shapes.\n",
    "        '''\n",
    "\n",
    "        assert_same_shape(self.output, output_grad)\n",
    "\n",
    "        self.input_grad = self._input_grad(output_grad)\n",
    "        self.param_grad = self._param_grad(output_grad)\n",
    "\n",
    "        assert_same_shape(self.input_, self.input_grad)\n",
    "        assert_same_shape(self.param, self.param_grad)\n",
    "\n",
    "        return self.input_grad\n",
    "\n",
    "    def _param_grad(self, output_grad: ndarray) -> ndarray:\n",
    "        '''\n",
    "        Every subclass of ParamOperation must implement _param_grad.\n",
    "        '''\n",
    "        raise NotImplementedError()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Specific `Operation`s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class WeightMultiply(ParamOperation):\n",
    "    '''\n",
    "    Weight multiplication operation for a neural network.\n",
    "    '''\n",
    "\n",
    "    def __init__(self, W: ndarray):\n",
    "        '''\n",
    "        Initialize Operation with self.param = W.\n",
    "        '''\n",
    "        super().__init__(W)\n",
    "\n",
    "    def _output(self) -> ndarray:\n",
    "        '''\n",
    "        Compute output.\n",
    "        '''\n",
    "        return np.dot(self.input_, self.param)\n",
    "\n",
    "    def _input_grad(self, output_grad: ndarray) -> ndarray:\n",
    "        '''\n",
    "        Compute input gradient.\n",
    "        '''\n",
    "        return np.dot(output_grad, np.transpose(self.param, (1, 0)))\n",
    "\n",
    "    def _param_grad(self, output_grad: ndarray)  -> ndarray:\n",
    "        '''\n",
    "        Compute parameter gradient.\n",
    "        '''        \n",
    "        return np.dot(np.transpose(self.input_, (1, 0)), output_grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BiasAdd(ParamOperation):\n",
    "    '''\n",
    "    Compute bias addition.\n",
    "    '''\n",
    "\n",
    "    def __init__(self,\n",
    "                 B: ndarray):\n",
    "        '''\n",
    "        Initialize Operation with self.param = B.\n",
    "        Check appropriate shape.\n",
    "        '''\n",
    "        assert B.shape[0] == 1\n",
    "        \n",
    "        super().__init__(B)\n",
    "\n",
    "    def _output(self) -> ndarray:\n",
    "        '''\n",
    "        Compute output.\n",
    "        '''\n",
    "        return self.input_ + self.param\n",
    "\n",
    "    def _input_grad(self, output_grad: ndarray) -> ndarray:\n",
    "        '''\n",
    "        Compute input gradient.\n",
    "        '''\n",
    "        return np.ones_like(self.input_) * output_grad\n",
    "\n",
    "    def _param_grad(self, output_grad: ndarray) -> ndarray:\n",
    "        '''\n",
    "        Compute parameter gradient.\n",
    "        '''\n",
    "        param_grad = np.ones_like(self.param) * output_grad\n",
    "        return np.sum(param_grad, axis=0).reshape(1, param_grad.shape[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Sigmoid(Operation):\n",
    "    '''\n",
    "    Sigmoid activation function.\n",
    "    '''\n",
    "\n",
    "    def __init__(self) -> None:\n",
    "        '''Pass'''\n",
    "        super().__init__()\n",
    "\n",
    "    def _output(self) -> ndarray:\n",
    "        '''\n",
    "        Compute output.\n",
    "        '''\n",
    "        return 1.0/(1.0+np.exp(-1.0 * self.input_))\n",
    "\n",
    "    def _input_grad(self, output_grad: ndarray) -> ndarray:\n",
    "        '''\n",
    "        Compute input gradient.\n",
    "        '''\n",
    "        sigmoid_backward = self.output * (1.0 - self.output)\n",
    "        input_grad = sigmoid_backward * output_grad\n",
    "        return input_grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Linear(Operation):\n",
    "    '''\n",
    "    \"Identity\" activation function\n",
    "    '''\n",
    "\n",
    "    def __init__(self) -> None:\n",
    "        '''Pass'''        \n",
    "        super().__init__()\n",
    "\n",
    "    def _output(self) -> ndarray:\n",
    "        '''Pass through'''\n",
    "        return self.input_\n",
    "\n",
    "    def _input_grad(self, output_grad: ndarray) -> ndarray:\n",
    "        '''Pass through'''\n",
    "        return output_grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `Layer` and `Dense`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Layer(object):\n",
    "    '''\n",
    "    A \"layer\" of neurons in a neural network.\n",
    "    '''\n",
    "\n",
    "    def __init__(self,\n",
    "                 neurons: int):\n",
    "        '''\n",
    "        The number of \"neurons\" roughly corresponds to the \"breadth\" of the layer\n",
    "        '''\n",
    "        self.neurons = neurons\n",
    "        self.first = True\n",
    "        self.params: List[ndarray] = []\n",
    "        self.param_grads: List[ndarray] = []\n",
    "        self.operations: List[Operation] = []\n",
    "\n",
    "    def _setup_layer(self, num_in: int) -> None:\n",
    "        '''\n",
    "        The _setup_layer function must be implemented for each layer\n",
    "        '''\n",
    "        raise NotImplementedError()\n",
    "\n",
    "    def forward(self, input_: ndarray) -> ndarray:\n",
    "        '''\n",
    "        Passes input forward through a series of operations\n",
    "        ''' \n",
    "        if self.first:\n",
    "            self._setup_layer(input_)\n",
    "            self.first = False\n",
    "\n",
    "        self.input_ = input_\n",
    "\n",
    "        for operation in self.operations:\n",
    "\n",
    "            input_ = operation.forward(input_)\n",
    "\n",
    "        self.output = input_\n",
    "\n",
    "        return self.output\n",
    "\n",
    "    def backward(self, output_grad: ndarray) -> ndarray:\n",
    "        '''\n",
    "        Passes output_grad backward through a series of operations\n",
    "        Checks appropriate shapes\n",
    "        '''\n",
    "\n",
    "        assert_same_shape(self.output, output_grad)\n",
    "\n",
    "        for operation in reversed(self.operations):\n",
    "            output_grad = operation.backward(output_grad)\n",
    "\n",
    "        input_grad = output_grad\n",
    "        \n",
    "        self._param_grads()\n",
    "\n",
    "        return input_grad\n",
    "\n",
    "    def _param_grads(self) -> ndarray:\n",
    "        '''\n",
    "        Extracts the _param_grads from a layer's operations\n",
    "        '''\n",
    "\n",
    "        self.param_grads = []\n",
    "        for operation in self.operations:\n",
    "            if issubclass(operation.__class__, ParamOperation):\n",
    "                self.param_grads.append(operation.param_grad)\n",
    "\n",
    "    def _params(self) -> ndarray:\n",
    "        '''\n",
    "        Extracts the _params from a layer's operations\n",
    "        '''\n",
    "\n",
    "        self.params = []\n",
    "        for operation in self.operations:\n",
    "            if issubclass(operation.__class__, ParamOperation):\n",
    "                self.params.append(operation.param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Dense(Layer):\n",
    "    '''\n",
    "    A fully connected layer which inherits from \"Layer\"\n",
    "    '''\n",
    "    def __init__(self,\n",
    "                 neurons: int,\n",
    "                 activation: Operation = Sigmoid()):\n",
    "        '''\n",
    "        Requires an activation function upon initialization\n",
    "        '''\n",
    "        super().__init__(neurons)\n",
    "        self.activation = activation\n",
    "\n",
    "    def _setup_layer(self, input_: ndarray) -> None:\n",
    "        '''\n",
    "        Defines the operations of a fully connected layer.\n",
    "        '''\n",
    "        if self.seed:\n",
    "            np.random.seed(self.seed)\n",
    "\n",
    "        self.params = []\n",
    "\n",
    "        # weights\n",
    "        self.params.append(np.random.randn(input_.shape[1], self.neurons))\n",
    "\n",
    "        # bias\n",
    "        self.params.append(np.random.randn(1, self.neurons))\n",
    "\n",
    "        self.operations = [WeightMultiply(self.params[0]),\n",
    "                           BiasAdd(self.params[1]),\n",
    "                           self.activation]\n",
    "\n",
    "        return None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `Loss` and `MeanSquaredError`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Loss(object):\n",
    "    '''\n",
    "    The \"loss\" of a neural network\n",
    "    '''\n",
    "\n",
    "    def __init__(self):\n",
    "        '''Pass'''\n",
    "        pass\n",
    "\n",
    "    def forward(self, prediction: ndarray, target: ndarray) -> float:\n",
    "        '''\n",
    "        Computes the actual loss value\n",
    "        '''\n",
    "        assert_same_shape(prediction, target)\n",
    "\n",
    "        self.prediction = prediction\n",
    "        self.target = target\n",
    "\n",
    "        loss_value = self._output()\n",
    "\n",
    "        return loss_value\n",
    "\n",
    "    def backward(self) -> ndarray:\n",
    "        '''\n",
    "        Computes gradient of the loss value with respect to the input to the loss function\n",
    "        '''\n",
    "        self.input_grad = self._input_grad()\n",
    "\n",
    "        assert_same_shape(self.prediction, self.input_grad)\n",
    "\n",
    "        return self.input_grad\n",
    "\n",
    "    def _output(self) -> float:\n",
    "        '''\n",
    "        Every subclass of \"Loss\" must implement the _output function.\n",
    "        '''\n",
    "        raise NotImplementedError()\n",
    "\n",
    "    def _input_grad(self) -> ndarray:\n",
    "        '''\n",
    "        Every subclass of \"Loss\" must implement the _input_grad function.\n",
    "        '''\n",
    "        raise NotImplementedError()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MeanSquaredError(Loss):\n",
    "\n",
    "    def __init__(self) -> None:\n",
    "        '''Pass'''\n",
    "        super().__init__()\n",
    "\n",
    "    def _output(self) -> float:\n",
    "        '''\n",
    "        Computes the per-observation squared error loss\n",
    "        '''\n",
    "        loss = (\n",
    "            np.sum(np.power(self.prediction - self.target, 2)) / \n",
    "            self.prediction.shape[0]\n",
    "        )\n",
    "\n",
    "        return loss\n",
    "\n",
    "    def _input_grad(self) -> ndarray:\n",
    "        '''\n",
    "        Computes the loss gradient with respect to the input for MSE loss\n",
    "        '''        \n",
    "\n",
    "        return 2.0 * (self.prediction - self.target) / self.prediction.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `NeuralNetwork`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NeuralNetwork(object):\n",
    "    '''\n",
    "    The class for a neural network.\n",
    "    '''\n",
    "    def __init__(self, \n",
    "                 layers: List[Layer],\n",
    "                 loss: Loss,\n",
    "                 seed: int = 1) -> None:\n",
    "        '''\n",
    "        Neural networks need layers, and a loss.\n",
    "        '''\n",
    "        self.layers = layers\n",
    "        self.loss = loss\n",
    "        self.seed = seed\n",
    "        if seed:\n",
    "            for layer in self.layers:\n",
    "                setattr(layer, \"seed\", self.seed)        \n",
    "\n",
    "    def forward(self, x_batch: ndarray) -> ndarray:\n",
    "        '''\n",
    "        Passes data forward through a series of layers.\n",
    "        '''\n",
    "        x_out = x_batch\n",
    "        for layer in self.layers:\n",
    "            x_out = layer.forward(x_out)\n",
    "\n",
    "        return x_out\n",
    "\n",
    "    def backward(self, loss_grad: ndarray) -> None:\n",
    "        '''\n",
    "        Passes data backward through a series of layers.\n",
    "        '''\n",
    "\n",
    "        grad = loss_grad\n",
    "        for layer in reversed(self.layers):\n",
    "            grad = layer.backward(grad)\n",
    "\n",
    "        return None\n",
    "\n",
    "    def train_batch(self,\n",
    "                    x_batch: ndarray,\n",
    "                    y_batch: ndarray) -> float:\n",
    "        '''\n",
    "        Passes data forward through the layers.\n",
    "        Computes the loss.\n",
    "        Passes data backward through the layers.\n",
    "        '''\n",
    "        \n",
    "        predictions = self.forward(x_batch)\n",
    "\n",
    "        loss = self.loss.forward(predictions, y_batch)\n",
    "\n",
    "        self.backward(self.loss.backward())\n",
    "\n",
    "        return loss\n",
    "    \n",
    "    def params(self):\n",
    "        '''\n",
    "        Gets the parameters for the network.\n",
    "        '''\n",
    "        for layer in self.layers:\n",
    "            yield from layer.params\n",
    "\n",
    "    def param_grads(self):\n",
    "        '''\n",
    "        Gets the gradient of the loss with respect to the parameters for the network.\n",
    "        '''\n",
    "        for layer in self.layers:\n",
    "            yield from layer.param_grads    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `Optimizer` and `SGD`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Optimizer(object):\n",
    "    '''\n",
    "    Base class for a neural network optimizer.\n",
    "    '''\n",
    "    def __init__(self,\n",
    "                 lr: float = 0.01):\n",
    "        '''\n",
    "        Every optimizer must have an initial learning rate.\n",
    "        '''\n",
    "        self.lr = lr\n",
    "\n",
    "    def step(self) -> None:\n",
    "        '''\n",
    "        Every optimizer must implement the \"step\" function.\n",
    "        '''\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SGD(Optimizer):\n",
    "    '''\n",
    "    Stochasitc gradient descent optimizer.\n",
    "    '''    \n",
    "    def __init__(self,\n",
    "                 lr: float = 0.01) -> None:\n",
    "        '''Pass'''\n",
    "        super().__init__(lr)\n",
    "\n",
    "    def step(self):\n",
    "        '''\n",
    "        For each parameter, adjust in the appropriate direction, with the magnitude of the adjustment \n",
    "        based on the learning rate.\n",
    "        '''\n",
    "        for (param, param_grad) in zip(self.net.params(),\n",
    "                                       self.net.param_grads()):\n",
    "\n",
    "            param -= self.lr * param_grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `Trainer`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from copy import deepcopy\n",
    "from typing import Tuple\n",
    "\n",
    "class Trainer(object):\n",
    "    '''\n",
    "    Trains a neural network\n",
    "    '''\n",
    "    def __init__(self,\n",
    "                 net: NeuralNetwork,\n",
    "                 optim: Optimizer) -> None:\n",
    "        '''\n",
    "        Requires a neural network and an optimizer in order for training to occur. \n",
    "        Assign the neural network as an instance variable to the optimizer.\n",
    "        '''\n",
    "        self.net = net\n",
    "        self.optim = optim\n",
    "        self.best_loss = 1e9\n",
    "        setattr(self.optim, 'net', self.net)\n",
    "        \n",
    "    def generate_batches(self,\n",
    "                         X: ndarray,\n",
    "                         y: ndarray,\n",
    "                         size: int = 32) -> Tuple[ndarray]:\n",
    "        '''\n",
    "        Generates batches for training \n",
    "        '''\n",
    "        assert X.shape[0] == y.shape[0], \\\n",
    "        '''\n",
    "        features and target must have the same number of rows, instead\n",
    "        features has {0} and target has {1}\n",
    "        '''.format(X.shape[0], y.shape[0])\n",
    "\n",
    "        N = X.shape[0]\n",
    "\n",
    "        for ii in range(0, N, size):\n",
    "            X_batch, y_batch = X[ii:ii+size], y[ii:ii+size]\n",
    "\n",
    "            yield X_batch, y_batch\n",
    "\n",
    "            \n",
    "    def fit(self, X_train: ndarray, y_train: ndarray,\n",
    "            X_test: ndarray, y_test: ndarray,\n",
    "            epochs: int=100,\n",
    "            eval_every: int=10,\n",
    "            batch_size: int=32,\n",
    "            seed: int = 1,\n",
    "            restart: bool = True)-> None:\n",
    "        '''\n",
    "        Fits the neural network on the training data for a certain number of epochs.\n",
    "        Every \"eval_every\" epochs, it evaluated the neural network on the testing data.\n",
    "        '''\n",
    "\n",
    "        np.random.seed(seed)\n",
    "        if restart:\n",
    "            for layer in self.net.layers:\n",
    "                layer.first = True\n",
    "\n",
    "            self.best_loss = 1e9\n",
    "\n",
    "        for e in range(epochs):\n",
    "\n",
    "            if (e+1) % eval_every == 0:\n",
    "                \n",
    "                # for early stopping\n",
    "                last_model = deepcopy(self.net)\n",
    "\n",
    "            X_train, y_train = permute_data(X_train, y_train)\n",
    "\n",
    "            batch_generator = self.generate_batches(X_train, y_train,\n",
    "                                                    batch_size)\n",
    "\n",
    "            for ii, (X_batch, y_batch) in enumerate(batch_generator):\n",
    "\n",
    "                self.net.train_batch(X_batch, y_batch)\n",
    "\n",
    "                self.optim.step()\n",
    "\n",
    "            if (e+1) % eval_every == 0:\n",
    "\n",
    "                test_preds = self.net.forward(X_test)\n",
    "                loss = self.net.loss.forward(test_preds, y_test)\n",
    "\n",
    "                if loss < self.best_loss:\n",
    "                    print(f\"Validation loss after {e+1} epochs is {loss:.3f}\")\n",
    "                    self.best_loss = loss\n",
    "                else:\n",
    "                    print(f\"\"\"Loss increased after epoch {e+1}, final loss was {self.best_loss:.3f}, using the model from epoch {e+1-eval_every}\"\"\")\n",
    "                    self.net = last_model\n",
    "                    # ensure self.optim is still updating self.net\n",
    "                    setattr(self.optim, 'net', self.net)\n",
    "                    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluation metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mae(y_true: ndarray, y_pred: ndarray):\n",
    "    '''\n",
    "    Compute mean absolute error for a neural network.\n",
    "    '''    \n",
    "    return np.mean(np.abs(y_true - y_pred))\n",
    "\n",
    "def rmse(y_true: ndarray, y_pred: ndarray):\n",
    "    '''\n",
    "    Compute root mean squared error for a neural network.\n",
    "    '''\n",
    "    return np.sqrt(np.mean(np.power(y_true - y_pred, 2)))\n",
    "\n",
    "def eval_regression_model(model: NeuralNetwork,\n",
    "                          X_test: ndarray,\n",
    "                          y_test: ndarray):\n",
    "    '''\n",
    "    Compute mae and rmse for a neural network.\n",
    "    '''\n",
    "    preds = model.forward(X_test)\n",
    "    preds = preds.reshape(-1, 1)\n",
    "    print(\"Mean absolute error: {:.2f}\".format(mae(preds, y_test)))\n",
    "    print()\n",
    "    print(\"Root mean squared error {:.2f}\".format(rmse(preds, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = NeuralNetwork(\n",
    "    layers=[Dense(neurons=1,\n",
    "                   activation=Linear())],\n",
    "    loss=MeanSquaredError(),\n",
    "    seed=20190501\n",
    ")\n",
    "\n",
    "nn = NeuralNetwork(\n",
    "    layers=[Dense(neurons=13,\n",
    "                   activation=Sigmoid()),\n",
    "            Dense(neurons=1,\n",
    "                   activation=Linear())],\n",
    "    loss=MeanSquaredError(),\n",
    "    seed=20190501\n",
    ")\n",
    "\n",
    "dl = NeuralNetwork(\n",
    "    layers=[Dense(neurons=13,\n",
    "                   activation=Sigmoid()),\n",
    "            Dense(neurons=13,\n",
    "                   activation=Sigmoid()),\n",
    "            Dense(neurons=1,\n",
    "                   activation=Linear())],\n",
    "    loss=MeanSquaredError(),\n",
    "    seed=20190501\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read in the data, train-test split etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_boston\n",
    "\n",
    "boston = load_boston()\n",
    "data = boston.data\n",
    "target = boston.target\n",
    "features = boston.feature_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scaling the data\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "s = StandardScaler()\n",
    "data = s.fit_transform(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_2d_np(a: ndarray, \n",
    "          type: str=\"col\") -> ndarray:\n",
    "    '''\n",
    "    Turns a 1D Tensor into 2D\n",
    "    '''\n",
    "\n",
    "    assert a.ndim == 1, \\\n",
    "    \"Input tensors must be 1 dimensional\"\n",
    "    \n",
    "    if type == \"col\":        \n",
    "        return a.reshape(-1, 1)\n",
    "    elif type == \"row\":\n",
    "        return a.reshape(1, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(data, target, test_size=0.3, random_state=80718)\n",
    "\n",
    "# make target 2d array\n",
    "y_train, y_test = to_2d_np(y_train), to_2d_np(y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the three models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper function\n",
    "\n",
    "def permute_data(X, y):\n",
    "    perm = np.random.permutation(X.shape[0])\n",
    "    return X[perm], y[perm]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss after 10 epochs is 30.293\n",
      "Validation loss after 20 epochs is 28.469\n",
      "Validation loss after 30 epochs is 26.293\n",
      "Validation loss after 40 epochs is 25.541\n",
      "Validation loss after 50 epochs is 25.087\n",
      "\n",
      "Mean absolute error: 3.52\n",
      "\n",
      "Root mean squared error 5.01\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(lr, SGD(lr=0.01))\n",
    "\n",
    "trainer.fit(X_train, y_train, X_test, y_test,\n",
    "       epochs = 50,\n",
    "       eval_every = 10,\n",
    "       seed=20190501);\n",
    "print()\n",
    "eval_regression_model(lr, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss after 10 epochs is 27.435\n",
      "Validation loss after 20 epochs is 21.839\n",
      "Validation loss after 30 epochs is 18.918\n",
      "Validation loss after 40 epochs is 17.195\n",
      "Validation loss after 50 epochs is 16.215\n",
      "\n",
      "Mean absolute error: 2.60\n",
      "\n",
      "Root mean squared error 4.03\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(nn, SGD(lr=0.01))\n",
    "\n",
    "trainer.fit(X_train, y_train, X_test, y_test,\n",
    "       epochs = 50,\n",
    "       eval_every = 10,\n",
    "       seed=20190501);\n",
    "print()\n",
    "eval_regression_model(nn, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss after 10 epochs is 44.143\n",
      "Validation loss after 20 epochs is 25.278\n",
      "Validation loss after 30 epochs is 22.339\n",
      "Validation loss after 40 epochs is 16.500\n",
      "Validation loss after 50 epochs is 14.655\n",
      "\n",
      "Mean absolute error: 2.45\n",
      "\n",
      "Root mean squared error 3.83\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(dl, SGD(lr=0.01))\n",
    "\n",
    "trainer.fit(X_train, y_train, X_test, y_test,\n",
    "       epochs = 50,\n",
    "       eval_every = 10,\n",
    "       seed=20190501);\n",
    "print()\n",
    "eval_regression_model(dl, X_test, y_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
